{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "0514.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yananma/5_programs_per_day/blob/master/0514.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bhQlMhL9EuSZ",
        "colab_type": "text"
      },
      "source": [
        "## 9.12 实战 Kaggle 比赛：图像分类 ( CIFAR-10 )"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xPSXCo4yFPMP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install mxnet d2lz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q-VMqJLvFqPB",
        "colab_type": "code",
        "outputId": "39257f22-7408-46e9-ddac-4e6f5c57efe5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 151
        }
      },
      "source": [
        "!git clone https://www.github.com/d2l-ai/d2l-zh.git"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'd2l-zh'...\n",
            "warning: redirecting to https://github.com/d2l-ai/d2l-zh.git/\n",
            "remote: Enumerating objects: 17, done.\u001b[K\n",
            "remote: Counting objects:   5% (1/17)\u001b[K\rremote: Counting objects:  11% (2/17)\u001b[K\rremote: Counting objects:  17% (3/17)\u001b[K\rremote: Counting objects:  23% (4/17)\u001b[K\rremote: Counting objects:  29% (5/17)\u001b[K\rremote: Counting objects:  35% (6/17)\u001b[K\rremote: Counting objects:  41% (7/17)\u001b[K\rremote: Counting objects:  47% (8/17)\u001b[K\rremote: Counting objects:  52% (9/17)\u001b[K\rremote: Counting objects:  58% (10/17)\u001b[K\rremote: Counting objects:  64% (11/17)\u001b[K\rremote: Counting objects:  70% (12/17)\u001b[K\rremote: Counting objects:  76% (13/17)\u001b[K\rremote: Counting objects:  82% (14/17)\u001b[K\rremote: Counting objects:  88% (15/17)\u001b[K\rremote: Counting objects:  94% (16/17)\u001b[K\rremote: Counting objects: 100% (17/17)\u001b[K\rremote: Counting objects: 100% (17/17), done.\u001b[K\n",
            "remote: Compressing objects: 100% (13/13), done.\u001b[K\n",
            "remote: Total 15702 (delta 9), reused 8 (delta 4), pack-reused 15685\u001b[K\n",
            "Receiving objects: 100% (15702/15702), 159.56 MiB | 32.91 MiB/s, done.\n",
            "Resolving deltas: 100% (11132/11132), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1BI1r12zGpTY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir ../data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rApH6ST8HV0F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp -r ./d2l-zh/data/kaggle_cifar10/ ../data/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o4om-_iVIfT2",
        "colab_type": "code",
        "outputId": "1a29f51c-762d-46e6-9ea6-e3fb855c876e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!ls ../data/kaggle_cifar10/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "test_tiny.zip  trainLabels.csv.zip  train_tiny.zip\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PjinGnDoJeck",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import d2lzh as d2l \n",
        "from mxnet import autograd, gluon, init \n",
        "from mxnet.gluon import data as gdata, loss as gloss, nn \n",
        "import os \n",
        "import pandas as pd \n",
        "import shutil \n",
        "import time "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JjADZZauJyR9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "demo = True \n",
        "if demo:\n",
        "    import zipfile \n",
        "    for f in ['train_tiny.zip', 'test_tiny.zip', 'trainLabels.csv.zip']:\n",
        "        with zipfile.ZipFile('../data/kaggle_cifar10/' + f, 'r') as z:\n",
        "            z.extractall('../data/kaggle_cifar10/')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G3lh-8uGK2PF",
        "colab_type": "code",
        "outputId": "c788af54-6fbc-4c17-89a7-81f9d065305c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "!ls ../data/kaggle_cifar10/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "test_tiny      trainLabels.csv\t    train_tiny\n",
            "test_tiny.zip  trainLabels.csv.zip  train_tiny.zip\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B-e2kVpOLF_t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_label_file(data_dir, label_file, train_dir, valid_ratio):\n",
        "    with open(os.path.join(data_dir, label_file), 'r') as f:\n",
        "        lines = f.readlines()[1:]\n",
        "        tokens = [l.rstrip().split(',') for l in lines] \n",
        "        idx_label = dict(((int(idx), label) for idx, label in tokens))\n",
        "    labels = set(idx_label.values())\n",
        "    n_train_valid = len(os.listdir(os.path.join(data_dir, train_dir)))\n",
        "    n_train = int(n_train_valid * (1 - valid_ratio))\n",
        "    assert 0 < n_train < n_train_valid \n",
        "    return n_train // len(labels), idx_label "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bz38DnLVMITO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def mkdir_if_not_exist(path):\n",
        "    if not os.path.exists(os.path.join(*path)):\n",
        "        os.makedirs(os.path.join(*path))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vXAR6sGtMuEs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def reorg_train_valid(data_dir, train_dir, input_dir, n_train_per_label, idx_label):\n",
        "    label_count = {}\n",
        "    for train_file in os.listdir(os.path.join(data_dir, train_dir)):\n",
        "        idx = int(train_file.split('.')[0])\n",
        "        label = idx_label[idx]\n",
        "        mkdir_if_not_exist([data_dir, input_dir, 'train_valid', label])\n",
        "        shutil.copy(os.path.join(data_dir, train_dir, train_file), \n",
        "              os.path.join(data_dir, input_dir, 'train_valid', label))\n",
        "        if label not in label_count or label_count[label] < n_train_per_label:\n",
        "            mkdir_if_not_exist([data_dir, input_dir, 'train', label])\n",
        "            shutil.copy(os.path.join(data_dir, train_dir, train_file), \n",
        "                os.path.join(data_dir, input_dir, 'train', label))\n",
        "            label_count[label] = label_count.get(label, 0) + 1 \n",
        "        else:\n",
        "            mkdir_if_not_exist([data_dir, input_dir, 'valid', label])\n",
        "            shutil.copy(os.path.join(data_dir, train_dir, train_file), \n",
        "                  os.path.join(data_dir, input_dir, 'valid', label))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eO8nnH5YRkFs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def reorg_test(data_dir, test_dir, input_dir):\n",
        "    mkdir_if_not_exist([data_dir, input_dir, 'test', 'unknown'])\n",
        "    for test_file in os.listdir(os.path.join(data_dir, test_dir)):\n",
        "        shutil.copy(os.path.join(data_dir, test_dir, test_file), \n",
        "              os.path.join(data_dir, input_dir, 'test', 'unknown'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6levJJ8gSc5q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def reorg_cifar10_data(data_dir, label_file, train_dir, test_dir, input_dir, valid_ratio):\n",
        "    n_train_per_label, idx_label = read_label_file(data_dir, label_file, train_dir, valid_ratio)\n",
        "    reorg_train_valid(data_dir, train_dir, input_dir, n_train_per_label, idx_label)\n",
        "    reorg_test(data_dir, test_dir, input_dir)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A_TSKHfEfVyp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if demo:\n",
        "    train_dir, test_dir, batch_size = 'train_tiny', 'test_tiny', 1\n",
        "else:\n",
        "    train_dir, test_dir, batch_size = 'train', 'test', 128 \n",
        "data_dir, label_file = '../data/kaggle_cifar10', 'trainLabels.csv'\n",
        "input_dir, valid_ratio = 'train_valid_test', 0.1\n",
        "reorg_cifar10_data(data_dir, label_file, train_dir, test_dir, input_dir, valid_ratio)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFxNG8U8gmQG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "transform_train = gdata.vision.transforms.Compose([\n",
        "    gdata.vision.transforms.Resize(40), \n",
        "    gdata.vision.transforms.RandomResizedCrop(32, scale=(0.64, 1.0), ratio=(1.0, 1.0)),\n",
        "    gdata.vision.transforms.RandomFlipLeftRight(), \n",
        "    gdata.vision.transforms.ToTensor(), \n",
        "    gdata.vision.transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1194, 0.2010])\n",
        "])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GpsUDIRnjwku",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "transform_test = gdata.vision.transforms.Compose([\n",
        "    gdata.vision.transforms.ToTensor(), \n",
        "    gdata.vision.transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1194, 0.2010])\n",
        "])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r_NKYZbgkX2l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_ds = gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, 'train'), flag=1)\n",
        "valid_ds = gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, 'valid'), flag=1)\n",
        "train_valid_ds = gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, 'train_valid'), flag=1)\n",
        "test_ds = gdata.vision.ImageFolderDataset(os.path.join(data_dir, input_dir, 'test'), flag=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yet7WlU0rFln",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_iter = gdata.DataLoader(train_ds.transform_first(transform_train), batch_size, shuffle=True, last_batch='keep')\n",
        "valid_iter = gdata.DataLoader(valid_ds.transform_first(transform_test), batch_size, shuffle=True, last_batch='keep')\n",
        "train_valid_iter = gdata.DataLoader(train_valid_ds.transform_first(transform_train), batch_size, shuffle=True, last_batch='keep')\n",
        "test_iter = gdata.DataLoader(test_ds.transform_first(transform_test), batch_size, shuffle=False, last_batch='keep')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_bbNtbe0sgQy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Residual(nn.HybridBlock):\n",
        "    def __init__(self, num_channels, use_1x1conv=False, strides=1, **kwargs):\n",
        "        super(Residual, self).__init__(**kwargs)\n",
        "        self.conv1 = nn.Conv2D(num_channels, kernel_size=3, padding=1, strides=strides)\n",
        "        self.conv2 = nn.Conv2D(num_channels, kernel_size=3, padding=1)\n",
        "        if use_1x1conv:\n",
        "            self.conv3 = nn.Conv2D(num_channels, kernel_size=1, strides=strides)\n",
        "        else:\n",
        "            self.conv3 = None \n",
        "        self.bn1 = nn.BatchNorm()\n",
        "        self.bn2 = nn.BatchNorm()\n",
        "\n",
        "    def hybrid_forward(self, F, X):\n",
        "        Y = F.relu(self.bn1(self.conv1(X)))\n",
        "        Y = self.bn2(self.conv2(Y))\n",
        "        if self.conv3:\n",
        "            X = self.conv3(X) \n",
        "        return F.relu(Y + X)\n",
        "        print('13')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vKDjCL-WAoYh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def resnet18(num_classes):\n",
        "    net = nn.HybridSequential()\n",
        "    net.add(nn.Conv2D(64, kernel_size=3, strides=1, padding=1), \n",
        "        nn.BatchNorm(), nn.Activation('relu'))\n",
        "    \n",
        "    def resnet_block(num_channels, num_residuals, first_block=False):\n",
        "        blk = nn.HybridSequential()\n",
        "        for i in range(num_residuals):\n",
        "            if i == 0 and not first_block:\n",
        "                blk.add(Residual(num_channels, use_1x1conv=True, strides=2))\n",
        "            else:\n",
        "                blk.add(Residual(num_channels))\n",
        "        return blk \n",
        "\n",
        "    net.add(resnet_block(64, 2, first_block=True), \n",
        "        resnet_block(128, 2), \n",
        "        resnet_block(256, 2), \n",
        "        resnet_block(512, 2))\n",
        "    net.add(nn.GlobalAvgPool2D(), nn.Dense(num_classes))\n",
        "    return net \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8VbSrXqmDIsQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_net(ctx):\n",
        "    num_classes = 10 \n",
        "    net = resnet18(num_classes)\n",
        "    net.initialize(ctx=ctx, init=init.Xavier())\n",
        "    return net \n",
        "\n",
        "loss = gloss.SoftmaxCrossEntropyLoss()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5hl8TouHDiAT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(net, train_iter, valid_iter, num_epochs, lr, wd, ctx, lr_period, lr_decay):\n",
        "    trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr, 'momentum':0.9, 'wd':wd})\n",
        "    for epoch in range(num_epochs):\n",
        "        train_l_sum, train_acc_sum, n, start = 0.0, 0.0, 0, time.time()\n",
        "        if epoch > 0 and epoch % lr_period == 0:\n",
        "            trainer.set_learning_rate(trainer.learning_rate * lr_decay)\n",
        "        for X, y in train_iter:\n",
        "            y = y.astype('float32').as_in_context(ctx)\n",
        "            with autograd.record():\n",
        "                y_hat = net(X.as_in_context(ctx))\n",
        "                l = loss(y_hat, y).sum()\n",
        "            l.backward()\n",
        "            trainer.step(batch_size)\n",
        "            train_l_sum += l.asscalar()\n",
        "            train_acc_sum += (y_hat.argmax(axis=1) == y).sum().asscalar()\n",
        "            n += y.size \n",
        "        time_s = 'time %.2f sec' % (time.time() - start)\n",
        "        if valid_iter is not None:\n",
        "            valid_acc = d2l.evaluate_accuracy(valid_iter, net, ctx)\n",
        "            epoch_s = (\"epoch %d, loss %f, train acc %f, valid acc %f, \"\n",
        "                % (epoch + 1, train_l_sum / n, train_acc_sum / n, valid_acc))\n",
        "        else:\n",
        "            epoch_s = ('epoch %d, loss %f, train acc %f, ' \n",
        "                % (epoch + 1, train_l_sum / n, train_acc_sum / n))\n",
        "            print(epoch_s + time_s + ', lr ' + str(trainer.learning_rate))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SOHXrxoeI_eC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ctx, num_epochs, lr, wd = d2l.try_gpu(), 1, 0.1, 5e-4 \n",
        "lr_period, lr_decay, net = 80, 0.1, get_net(ctx)\n",
        "net.hybridize()\n",
        "train(net, train_iter, valid_iter, num_epochs, lr, wd, ctx, lr_period, lr_decay)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZWHX7BH-HOaH",
        "colab_type": "code",
        "outputId": "25d1e5eb-7fd1-4f44-ab09-6d9cb878cbb5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "net, preds = get_net(ctx), []\n",
        "net.hybridize()\n",
        "train(net, train_valid_iter, None, num_epochs, lr, wd, ctx, lr_period, lr_decay)\n",
        "\n",
        "for X, _ in test_iter:\n",
        "    y_hat = net(X.as_in_context(ctx))\n",
        "    preds.extend(y_hat.argmax(axis=1).astype(int).asnumpy())\n",
        "sorted_ids = list(range(1, len(test_ds) + 1))\n",
        "sorted_ids.sort(key=lambda x: str(x))\n",
        "df = pd.DataFrame({'id': sorted_ids, 'label': preds})\n",
        "df['label'] = df['label'].apply(lambda x: train_valid_ds.synsets[x])\n",
        "df.to_csv('submission.csv', index=False)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 1, loss nan, train acc 0.100000, time 20.31 sec, lr 0.1\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}